import json
import pandas as pd
import torch
from datasets import Dataset
from modelscope import snapshot_download, AutoTokenizer
from swanlab.integration.huggingface import SwanLabCallback
from peft import LoraConfig, TaskType, get_peft_model
from transformers import AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForSeq2Seq
import os
import swanlab

from config import OPERATE_MODEL_CONFIG


def dataset_jsonl_transfer(origin_path, new_path):
    """
    将原始数据集转换为大模型微调所需数据格式的新数据集
    """
    messages = []
    with open(origin_path, "r") as file:
        # 读取span标签配置
        with open("span/label_config.json", "r") as fileSpan:
            data = json.load(fileSpan)
            match_names = [item['text'] for item in data]
            print(match_names)
        for line in file:
            # 解析每一行的json数据
            data = json.loads(line)
            input_text = data["text"]
            # cats = data["cats"]
            entities = data["entities"]
            # match_names = ["S任务编号", "S工作站", "S订单编号"]
            entity_sentence = []
            for entity in entities:
                entity_text = input_text[entity[0]:entity[1]]
                entity_name = entity[2]
                for match_name in match_names:
                    if match_name == entity_name:
                        entity_label = entity_name
                        break
                entity_sentence.append({"entity_text": entity_text, "entity_label": entity_label})
            # if not entity_sentence:
            #     entity_sentence = "没有找到任何实体"
            output = json.dumps(entity_sentence, ensure_ascii=False)
            message = {
                "instruction": "你是一个文本实体识别领域的专家，你需要从给定的句子中提取 " + "; ".join(
                    match_names) + " 实体. 以 json数组 格式输出, 如 " + output + """ 注意: 1. 输出的每一行都必须是正确的 json数组 字符串 . 2. 找不到任何实体时, 输出"没有找到任何实体".""",
                "input": f"文本:{input_text}",
                # "input": f"类型:{cats[0]};文本:{input_text}",
                "output": output,
            }

            messages.append(message)

    # 保存重构后的JSONL文件
    with open(new_path, "w", encoding="utf-8") as file:
        for message in messages:
            file.write(json.dumps(message, ensure_ascii=False) + "\n")


def process_func(example):
    # 读取span标签配置
    with open("span/label_config.json", "r") as fileSpan:
        data = json.load(fileSpan)
        match_names = [item['text'] for item in data]
    print(match_names)
    """
    将数据集进行预处理, 处理成模型可以接受的格式
    """

    MAX_LENGTH = 384
    input_ids, attention_mask, labels = [], [], []
    system_prompt = "你是一个文本实体识别领域的专家，你需要从给定的句子中提取 " + "; ".join(
        match_names) + """ 实体. 以 json数组 格式输出, 如 [{"entity_text": "TASK202400000501", "entity_label": "任务编号"}, {"entity_text": "任务表", "entity_label": "表名"}] 注意: 1. 输出的每一行都必须是正确的 json数组 字符串 . 2. 找不到任何实体时, 输出"没有找到任何实体".""",

    instruction = tokenizer(
        f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n{example['input']}<|im_end|>\n<|im_start|>assistant\n",
        add_special_tokens=False,
    )
    response = tokenizer(f"{example['output']}", add_special_tokens=False)
    input_ids = instruction["input_ids"] + response["input_ids"] + [tokenizer.pad_token_id]
    attention_mask = (
            instruction["attention_mask"] + response["attention_mask"] + [1]
    )
    labels = [-100] * len(instruction["input_ids"]) + response["input_ids"] + [tokenizer.pad_token_id]
    if len(input_ids) > MAX_LENGTH:  # 做一个截断
        input_ids = input_ids[:MAX_LENGTH]
        attention_mask = attention_mask[:MAX_LENGTH]
        labels = labels[:MAX_LENGTH]
    return {"input_ids": input_ids, "attention_mask": attention_mask, "labels": labels}


def predict(messages, model, tokenizer):
    device = "cuda"
    text = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True
    )
    model_inputs = tokenizer([text], return_tensors="pt").to(device)

    generated_ids = model.generate(
        model_inputs.input_ids,
        max_new_tokens=512
    )
    generated_ids = [
        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
    ]

    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]

    print(response)

    return response


# model_id_path = "qwen/Qwen2-0.5B-Instruct"
# model_dir_path = "./qwen/Qwen2-0___5B-Instruct"
model_id_path = OPERATE_MODEL_CONFIG['model_id_path']
model_dir_path = OPERATE_MODEL_CONFIG['model_dir_path']

# 在modelscope上下载Qwen模型到本地目录下
model_dir = snapshot_download(model_id_path, cache_dir="./", revision="master")

# Transformers加载模型权重
tokenizer = AutoTokenizer.from_pretrained(model_dir, use_fast=False, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", torch_dtype=torch.bfloat16)
model.enable_input_require_grads()  # 开启梯度检查点时，要执行该方法

# 加载、处理数据集和测试集
train_dataset_path = "all.jsonl"
train_jsonl_new_path = "new_all_ner.jsonl"

if not os.path.exists(train_jsonl_new_path):
    dataset_jsonl_transfer(train_dataset_path, train_jsonl_new_path)

# 得到训练集
total_df = pd.read_json(train_jsonl_new_path, lines=True)
train_df = total_df[int(len(total_df) * 0.1):]
train_ds = Dataset.from_pandas(train_df)
train_dataset = train_ds.map(process_func, remove_columns=train_ds.column_names)

config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
    inference_mode=False,  # 训练模式
    r=8,  # Lora 秩
    lora_alpha=32,  # Lora alaph，具体作用参见 Lora 原理
    lora_dropout=0.1,  # Dropout 比例
)

model = get_peft_model(model, config)

# 定义训练参数
args = TrainingArguments(
    output_dir="./output/Qwen2-NER",  # 指定训练和保存模型的目录
    per_device_train_batch_size=48,  # 大幅增加每个设备上的训练批次大小
    per_device_eval_batch_size=48,  # 同样增加每个设备上的评估批次大小
    gradient_accumulation_steps=1,  # 将梯度累积步数调整为1，以匹配增大的批次大小
    logging_steps=100,  # 增加日志记录步数，减少频繁记录日志带来的开销
    num_train_epochs=3,  # 总训练轮次（保持不变）
    save_steps=1000,  # 增加保存步骤间隔，减少频繁保存带来的开销
    learning_rate=2e-5,  # 减小学习率以配合更大的批次大小
    save_total_limit=2,  # 仅保留最近的两个检查点，减少存储开销
    evaluation_strategy="steps",  # 评估策略
    eval_steps=1000,  # 每 1000 步进行一次评估
    fp16=True,  # 启用半精度训练，节省显存并加速训练过程
    gradient_checkpointing=True,  # 启用梯度检查点以节省显存
    report_to="none",  # 不报告训练日志到任何地方
    dataloader_num_workers=8,  # 增加数据加载的工作线程数，提高数据加载速度
)

swanlab_callback = SwanLabCallback(
    project="Qwen2-NER-fintune",
    experiment_name="Qwen2-0.5B-Instruct",
    description="使用通义千问Qwen2-0.5B-Instruct模型在NER数据集上微调，实现关键实体识别任务。",
    config={
        "model": model_id_path,
        "model_dir": model_dir_path,
        "dataset": "qgyd2021/chinese_ner_sft",
    },
)

trainer = Trainer(
    model=model,
    args=args,
    train_dataset=train_dataset,
    data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer, padding=True),
    callbacks=[swanlab_callback],
)

trainer.train()

# 得到测试集
test_df = total_df[:int(len(total_df) * 1)].sample(n=3)

test_text_list = []
for index, row in test_df.iterrows():
    instruction = row['instruction']
    input_value = row['input']

    messages = [
        {"role": "system", "content": f"{instruction}"},
        {"role": "user", "content": f"{input_value}"}
    ]

    print("test:" + messages)
    response = predict(messages, model, tokenizer)
    messages.append({"role": "assistant", "content": f"{response}"})
    result_text = f"{messages[0]}\n\n{messages[1]}\n\n{messages[2]}"
    test_text_list.append(swanlab.Text(result_text, caption=response))

swanlab.log({"Prediction": test_text_list})
swanlab.finish()
